"""
Copyright (c) 2022 Huawei Technologies Co.,Ltd.

openGauss is licensed under Mulan PSL v2.
You can use this software according to the terms and conditions of the Mulan PSL v2.
You may obtain a copy of Mulan PSL v2 at:

          http://license.coscl.org.cn/MulanPSL2

THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
See the Mulan PSL v2 for more details.
"""
"""
Case Type   : 故障&可靠性测试
Case Name   : 延迟等待过程中kill -15 各节点进程
Create At   : 2021/03/03
@zou_jialiang0505328126
Description :
    1.设置延迟为5min
    2.切换备节点为级联备
    3.创建表
    4.等待30s，kill -15 主节点
    5.等待30s启动主节点
    6.查询各节点
    7.插入数据;
    8.等待30s kill -15 备节点
    9.等待30s start备节点
    10.查询各节点
    11.更新数据
    12.等待30s，kill -15 级联备
    13.等待30s, start级联备
    14.查询各节点
Expect      :
    1.设置成功
    2.切换成功
    3.建表成功
    4.停止成功
    5.重启成功
    6.主节点有数据，备节点5min后同步
    7.插入数据成功
    8.停止备节点成功
    9.重启成功
    10.主节点有数据，备节点5min后同步
    11.更新数据成功
    12.停止成功
    13.启动成功
    14.主节点有数据，备节点5min后同步
History     :
"""

import unittest
import time
import datetime
from yat.test import Node
from testcase.utils.Common import Common
from testcase.utils.CommonSH import CommonSH
from testcase.utils.CommonSH import RestartDbCluster
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger


class RecoveryDelay(unittest.TestCase):
    commonshpri = CommonSH('PrimaryDbUser')
    nodes_tuple = ('PrimaryDbUser', 'Standby1DbUser', 'Standby2DbUser')

    @RestartDbCluster(*nodes_tuple)
    def setUp(self):
        self.log = Logger()
        self.log.info('----Opengauss_Reliability_Dbsys_Case264.py 开始执行------')
        self.db_primary_root_node = Node(node='PrimaryRoot')
        self.db_standby_node =  \
            [Node(node='Standby1Root'), Node(node='Standby2Root')]
        self.comshsta = \
            [CommonSH('Standby1DbUser'), CommonSH('Standby2DbUser')]
        self.constant = Constant()
        self.com_pri = Common('PrimaryRoot')
        self.com_sta = [Common('Standby1Root'), Common('Standby2Root')]
        self.tb_name = 'tb_256'

        self.log.info('=====安装ntp服务=======')
        install_cmd = "yum install -y ntp && " \
                      "systemctl enable ntpd && systemctl start ntpd"
        for i in range(2):
            result = self.db_standby_node[i].sh(install_cmd).result()
            self.log.info(result)

        self.log.info('------同步集群时间--------')
        for i in range(2):
            current = self.db_primary_root_node.sh(
                "date \"+%m/%d/%Y %H:%M:%S\"").result()
            self.log.info(current)
            datecmd = f'date -s "{current}";hwclock --systohc;' \
                f'service ntpd stop;ntpdate ntp.api.bz;service ntpd start'
            self.log.info(datecmd)
            result = self.db_standby_node[i].sh(datecmd).result()
            self.log.info(result)

        self.log.info('--------等待主备一致------------')
        for i in range(2):
            result = self.comshsta[i].check_data_consistency()
            self.assertTrue(result)

    def test_recoverydelay(self):
        self.log.info('-----设置recovery_min_apply_delay=3min----')
        result = self.commonshpri.executeGsguc(
            'reload', self.constant.GSGUC_SUCCESS_MSG,
            'recovery_min_apply_delay=3min')
        self.assertTrue(result)

        self.log.info('-------将备节点切换为级联备---------')
        result = self.comshsta[0].buildStandby('-M cascade_standby')
        self.assertIn(self.constant.REBUILD_SUCCESS_MSG, result)
        result = self.comshsta[0].exec_refresh_conf()
        self.assertTrue(result)

        self.log.info('--------等待主备一致------------')
        result = self.comshsta[1].check_data_consistency()
        self.assertTrue(result)
        for i in range(90):
            result = self.commonshpri.check_cascade_standby_consistency()
            if result:
                break
            time.sleep(20)
        self.assertTrue(result)

        self.log.info('-----------------创建表----------------')
        sql = f"drop table if exists {self.tb_name};" \
            f"create table {self.tb_name}(i int, s char(10));"
        result = self.commonshpri.executDbSql(sql)
        self.log.info(result)
        self.assertIn(self.constant.TABLE_CREATE_SUCCESS, result)

        self.log.info('--------------等待30s，kill -15主节点-------------')
        time.sleep(30)
        result = self.com_pri.kill_pid(self.db_primary_root_node, '15')
        self.log.info(result)

        self.log.info('--------------查询各节点-----------------')
        sql = f"select sysdate;select * from {self.tb_name};"
        result_pri = self.commonshpri.executDbSql(sql)
        self.log.info(result_pri)
        flg = self.constant.FAILED_CONNECT_DB in result_pri \
              or '0 rows' in result_pri
        self.assertTrue(flg)
        for i in range(2):
            result = self.comshsta[i].executDbSql(sql)
            self.log.info(result)
            self.assertIn(self.constant.NOT_EXIST, result)
        time.sleep(200)
        result_pri = self.commonshpri.executDbSql(sql)
        self.log.info(result_pri)
        if self.constant.FAILED_CONNECT_DB in result:
            self.log.info('====主节点停止===')
            result = self.commonshpri.startDbInstance()
            self.assertIn(self.constant.RESTART_SUCCESS_MSG, result)
            time.sleep(60)
        for i in range(2):
            result = self.comshsta[i].executDbSql(sql)
            self.log.info(result)
            self.assertIn('(0 rows)', result)

        self.log.info('-----------------插入数据----------------')
        sql = f"insert into {self.tb_name} values (1, 'test');"
        result = self.commonshpri.executDbSql(sql)
        self.log.info(result)
        if self.constant.FAILED_CONNECT_DB in result:
            self.log.info('====主节点停止===')
            result = self.commonshpri.startDbInstance()
            self.assertIn(self.constant.RESTART_SUCCESS_MSG, result)
            time.sleep(60)
            result = self.commonshpri.executDbSql(sql)
            self.log.info(result)
        self.assertIn('INSERT', result)

        self.log.info('--------------等待30s stop备节点-------------')
        time.sleep(30)
        result = self.com_sta[1].kill_pid(self.db_standby_node[1], '15')
        self.log.info(result)

        self.log.info('--------------查询各节点------------------')
        status_result = self.commonshpri.getDbClusterStatus("status")
        sql = f"select sysdate;select * from {self.tb_name};"
        if not status_result:
            self.log.info('=====备节点停止，启动备节点====')
            result = self.comshsta[1].startDbInstance('standby')
            self.assertIn(self.constant.RESTART_SUCCESS_MSG, result)
            for i in range(2):
                result = self.comshsta[i].executDbSql(sql)
                self.log.info(result)
                if 0 == i:
                    self.assertNotIn('1 | test', result)
                else:
                    flg = '1 | test' in result or '0 rows' in result
                    self.assertTrue(flg)
        else:
            for i in range(2):
                result = self.comshsta[i].executDbSql(sql)
                self.log.info(result)
                if 0 == i:
                    self.assertIn('(0 rows)', result)
                else:
                    flg = 'could not send startup' in result \
                          or '(0 rows)' in result or \
                          self.constant.FAILED_CONNECT_DB in result
                    self.assertTrue(flg)
            time.sleep(200)
            status_result = self.commonshpri.getDbClusterStatus("status")
            if not status_result:
                self.log.info('=====备节点停止，启动备节点====')
                result = self.comshsta[1].startDbInstance('standby')
                self.assertIn(self.constant.RESTART_SUCCESS_MSG, result)
                for i in range(2):
                    result = self.comshsta[i].executDbSql(sql)
                    self.log.info(result)
                    if 0 == i:
                        self.assertIn('1 | test', result)
                    else:
                        flg = '1 | test' in result or '0 rows' in result
                        self.assertTrue(flg)

        self.log.info('-----------------更新数据----------------')
        sql = f"update {self.tb_name} set i=5;"
        result = self.commonshpri.executDbSql(sql)
        self.log.info(result)
        self.assertIn('UPDATE', result)

        self.log.info('--------------等待30s kill -15级联备-------------')
        time.sleep(30)
        result = self.com_sta[0].kill_pid(self.db_standby_node[0], '15')
        self.log.info(result)

        self.log.info('--------------查询各节点------------------')
        status_result = self.commonshpri.getDbClusterStatus("status")
        sql = f"select sysdate;select * from {self.tb_name};"
        if not status_result:
            self.log.info('=====级联备停止，启动级联备====')
            result = self.comshsta[0].startDbInstance('cascade_standby')
            self.assertIn(self.constant.RESTART_SUCCESS_MSG, result)
            for i in range(2):
                result = self.comshsta[i].executDbSql(sql)
                self.log.info(result)
                if 1 == i:
                    self.assertNotIn('5 | test', result)
                else:
                    flg = '1 | test' in result or '5 | test' in result
                    self.assertTrue(flg)
        else:
            for i in range(2):
                result = self.comshsta[i].executDbSql(sql)
                self.log.info(result)
                if 1 == i:
                    self.assertIn('1 | test', result)
                else:
                    flg = 'could not send startup' in result \
                          or '1 | test' in result or \
                          self.constant.FAILED_CONNECT_DB in result
                    self.assertTrue(flg)
            time.sleep(200)
            status_result = self.commonshpri.getDbClusterStatus("status")
            if not status_result:
                self.log.info('=====级联备停止，启动级联备====')
                result = self.comshsta[0].startDbInstance('cascade_standby')
                self.assertIn(self.constant.RESTART_SUCCESS_MSG, result)
                for i in range(2):
                    result = self.comshsta[i].executDbSql(sql)
                    self.log.info(result)
                    if 1 == i:
                        self.assertIn('5 | test', result)
                    else:
                        flg = '1 | test' in result or '5 | test' in result
                        self.assertTrue(flg)

    def tearDown(self):
        self.log.info('-----------------------------环境清理--------------------')
        self.log.info('-------------还原recovery_min_apply_delay----------')
        result = self.commonshpri.executeGsguc(
            'set', self.constant.GSGUC_SUCCESS_MSG,
            'recovery_min_apply_delay=0')
        self.log.info(result)

        self.log.info('--------还原集群-------')
        self.comshsta[0].buildStandby('-M standby')
        result = self.comshsta[0].exec_refresh_conf()
        self.log.info(result)

        self.log.info('-----------重启数据库-----------')
        result = self.commonshpri.stopDbCluster()
        self.log.info(result)
        result = self.commonshpri.startDbCluster()
        self.log.info(result)

        time.sleep(5)
        self.log.info('--------删除表-------')
        sql = f"drop table if exists {self.tb_name};"
        result = self.commonshpri.executDbSql(sql)
        self.log.info(result)
        self.log.info('---Opengauss_Reliability_Dbsys_Case264.py 执行结束-------')